import os

import pymongo
import sqlite3
import requests

from t66y import settings
from t66y.settings import not_trash_img

conn = sqlite3.connect('t66y.db')
c = conn.cursor()
mongo_client = pymongo.MongoClient(host='127.0.0.1', port=27017)
collection = mongo_client.t66y.article

page = 0

while True:
    print("page")
    records = collection.find({}).limit(100).skip(page * 100)
    count = 0
    for item in records:
        count += 1
        imgs = []
        for i in item['details']:
            j = i.get('imgs')
            if not j:
                continue
            for img in j:
                if not_trash_img(img):
                    imgs.append(img)

        for img in imgs:
            path = None
            if item and item['time']:
                date = item['time'].split(" ")[-1]
                if date:
                    path = date + "/" + str(item['index']) + "/" + img.split('/')[-1]
                else:
                    path = str(item['index']) + "/" + img.split('/')[-1]
            else:
                path = img.split('/')[-1]

            img_path = settings.IMAGES_STORE + "/" + path
            if path and os.path.exists(img_path):
                #print('pic already exists: ' + img_path)
                # size = os.path.getsize(img_path)
                # if size < 1000:
                    # try:
                    #     pic = requests.get(img, proxies={'http': 'http://127.0.0.1:58591',
                    #                                      'https': 'http://127.0.0.1:58591'}, timeout=30)
                    #     with open(img_path, "wb") as f:
                    #         f.write(pic.content)
                    #         print('pic exceed saved: ' + img_path)
                    # except Exception as e:
                    #     print(e)
                    #     print('pic error: ' + img)
                pass
            else:
                try:
                    pic = requests.get(img, proxies={'http': 'http://127.0.0.1:58591', 'https': 'http://127.0.0.1:58591'}, timeout=30)
                    path = os.path.dirname(img_path)
                    if not os.path.exists(path):
                        os.makedirs(path)
                    with open(img_path, "wb") as f:
                        f.write(pic.content)
                        print('pic saved: ' + img_path)
                except Exception as e:
                    print(e)
                    print('pic error: ' + img)

    if count == 0:
        break
    page += 1